# 读取
## method1
table = pd.read_table('CS-6449-2022-0053.txt',encoding = 'gbk',sep = ",",chunksize = 1000)

## method2
reader = pd.read_table('CS-6449-2022-0053.txt',encoding = 'gbk',sep = ",",iterator = True)

loop = True
chunkSize = 1000
chunks = []
while loop:
    try:
        chunk = reader.get_chunk(chunkSize)
        chunks.append(chunk)
    except StopIteration:
        loop = False
        print("Iterata is stopped")
df = pd.concat(chunks,ignore_index = True)

## method3
reader = pd.read_csv('xx.csv'chunksize = 10000,iterator = True)
df_result = pd.DataFrame()
for temp in reader:
    print(temp.shape)
    temp2 = pd.merge(df_label_channel[['seq','flagy']],temp,how = "inner",left_on = ["seq"],right_on = ["cus_num"])
    df_result = df_result.append(temp2)
del temp,temp2
df_label_channel.shape,df_result.shape

# 步长迭代法，选择融合评分系数最大组合。
from sklearn import metrics

# data中有需融合的两个评分score1和score2及标签flagy
lst_para = []
lst_ks = []
lst_auc = []
for i in np.arange(0,1,0.01):
    data['score_mix'] = round(data['score1'] * i + data['score2'] * (1-i))
    # 计算ks、auc
    temp_fpr,temp_tpr,temp_thresholds = metrics.roc_curve(data['flagy'],data['score_mix'])
    temp_ks = max(temp_fpr-temp_tpr)
    temp_auc = 1 - metrics.auc(temp_fpr,temp_tpr)

    lst_para.append(float(i))
    lst_ks.append(temp_ks)
    lst_auc.append(temp_auc)

data_para = pd.DataFrame({'parameter':lst_para,'ks':lst_ks,'auc':lst_auc})

# 内存情况
import psutil
import os
info = psutil.virtual_memory()
print('内存使用：',psutil.Process(os.getpid()).memory_info().rss)
print('总内存：',info.total)
print('内存占比：',info.percent)
print('CPU个数：',psutil.cpu_count())









